"""
Case Type   : ustore_DQL_SET_OPERATIONS
Case Name   : 子查询调优subquery—union/minus/union all混合使用
Create At   : 2022-05-12
Owner       : nanyang12
Description :
    1.创建表
    2.使用copy命令插入数据
    3.子查询调优subquery—union/minus/union all混合使用
    4.环境清理
Expect      :
    1.创建成功
    2.数据插入成功
    3.成功
    4.清理成功
History     :
"""

import os
import subprocess
import unittest

from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class Ustore01(unittest.TestCase):
    def setUp(self):
        self.log = Logger()
        self.log.info(f'----- {os.path.basename(__file__)} start-----')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.pri_root = Node('PrimaryRoot')
        self.com = Common()
        self.constant = Constant()
        self.table_name1 = 't_ustore_Cast_mult_0001_01'
        self.table_name2 = 't_ustore_Cast_mult_0001_02'
        self.table_name3 = 't_ustore_Cast_mult_0001_03'
        self.table_name4 = 't_ustore_Cast_mult_0001_04'
        self.table_name5 = 't_ustore_Cast_mult_0001_05'
        self.table_name6 = 't_ustore_Cast_mult_0001_06'
        self.table_name7 = 't_ustore_Cast_mult_0001_07'
        self.table_name8 = 't_ustore_Cast_mult_0001_08'
        self.copy_file1 = 'customer_address_33.txt'
        self.copy_file2 = 'store_33.txt'
        self.copy_file3 = 'call_center_33.txt'
        self.copy_file4 = 'web_site_33.txt'
        self.copy_file5 = 'item_33.txt'
        self.copy_file6 = 'customer_33.txt'
        self.copy_file7 = 'customer_address_33.txt'
        self.copy_file8 = 'store_sales_33.txt'
        self.cur_path_1 = os.path.join(macro.SCRIPTS_PATH, self.copy_file1)
        self.cur_path_2 = os.path.join(macro.SCRIPTS_PATH, self.copy_file2)
        self.cur_path_3 = os.path.join(macro.SCRIPTS_PATH, self.copy_file3)
        self.cur_path_4 = os.path.join(macro.SCRIPTS_PATH, self.copy_file4)
        self.cur_path_5 = os.path.join(macro.SCRIPTS_PATH, self.copy_file5)
        self.cur_path_6 = os.path.join(macro.SCRIPTS_PATH, self.copy_file6)
        self.cur_path_7 = os.path.join(macro.SCRIPTS_PATH, self.copy_file7)
        self.cur_path_8 = os.path.join(macro.SCRIPTS_PATH, self.copy_file8)
        self.parent_path = os.path.dirname(macro.DB_INSTANCE_PATH)
        self.target_path1 = os.path.join(self.parent_path, self.copy_file1)
        self.target_path2 = os.path.join(self.parent_path, self.copy_file2)
        self.target_path3 = os.path.join(self.parent_path, self.copy_file3)
        self.target_path4 = os.path.join(self.parent_path, self.copy_file4)
        self.target_path5 = os.path.join(self.parent_path, self.copy_file5)
        self.target_path6 = os.path.join(self.parent_path, self.copy_file6)
        self.target_path7 = os.path.join(self.parent_path, self.copy_file7)
        self.target_path8 = os.path.join(self.parent_path, self.copy_file8)

    def test_join(self):
        step_txt = '-----step1:创建表;expect: 创建成功-----'
        self.log.info(step_txt)
        cmd_sql = f"""drop table if exists {self.table_name1}  CASCADE ;
                    drop table if exists {self.table_name2}  CASCADE ;
                    drop table if exists {self.table_name3}  CASCADE ;
                    drop table if exists {self.table_name4}  CASCADE ;
                    drop table if exists {self.table_name5}  CASCADE ;
                    drop table if exists {self.table_name6}  CASCADE ;
                    drop table if exists {self.table_name7}  CASCADE ;
                    drop table if exists {self.table_name8}  CASCADE ;
        create table {self.table_name1}
        (
            ca_address_sk             integer               not null,
            ca_address_id             char(16)              not null,
            ca_street_number          char(10)                      ,
            ca_street_name            varchar(60)                   ,
            ca_street_type            char(15)                      ,
            ca_suite_number           char(10)                      ,
            ca_city                   varchar(60)                   ,
            ca_county                 varchar(30)                   ,
            ca_state                  char(2)                       ,
            ca_zip                    char(10)                      ,
            ca_country                varchar(20)                   ,
            ca_gmt_offset             decimal(5,2)                  ,
            ca_location_type          char(20)                     
        )  WITH (STORAGE_TYPE=USTORE);        
        create table {self.table_name2}
        (
            s_store_sk                integer               not null,
            s_store_id                char(16)              not null,
            s_rec_start_date          date                          ,
            s_rec_end_date            date                          ,
            s_closed_date_sk          integer                       ,
            s_store_name              varchar(50)                   ,
            s_number_employees        integer                       ,
            s_floor_space             integer                       ,
            s_hours                   char(20)                      ,
            s_manager                 varchar(40)                   ,
            s_market_id               integer                       ,
            s_geography_class         varchar(100)                  ,
            s_market_desc             varchar(100)                  ,
            s_market_manager          varchar(40)                   ,
            s_division_id             integer                       ,
            s_division_name           varchar(50)                   ,
            s_company_id              integer                       ,
            s_company_name            varchar(50)                   ,
            s_street_number           varchar(10)                   ,
            s_street_name             varchar(60)                   ,
            s_street_type             char(15)                      ,
            s_suite_number            char(10)                      ,
            s_city                    varchar(60)                   ,
            s_county                  varchar(30)                   ,
            s_state                   char(2)                       ,
            s_zip                     char(10)                      ,
            s_country                 varchar(20)                   ,
            s_gmt_offset              decimal(5,2)                  ,
            s_tax_precentage          decimal(5,2)                  
         )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name3}
        (
          cc_call_center_sk         integer               not null,
          cc_call_center_id         char(16)              not null,
          cc_rec_start_date         date                          ,
          cc_rec_end_date           date                          ,
          cc_closed_date_sk         integer                       ,
          cc_open_date_sk           integer                       ,
          cc_name                   varchar(50)                   ,
          cc_class                  varchar(50)                   ,
          cc_employees              integer                       ,
          cc_sq_ft                  integer                       ,
          cc_hours                  char(20)                      ,
          cc_manager                varchar(40)                   ,
          cc_mkt_id                 integer                       ,
          cc_mkt_class              char(50)                      ,
          cc_mkt_desc               varchar(100)                  ,
          cc_market_manager         varchar(100)                   ,
          cc_division               varchar(100)                       ,
          cc_division_name          varchar(50)                   ,
          cc_company                varchar(100)                       ,
          cc_company_name           char(50)                      ,
          cc_street_number          char(10)                      ,
          cc_street_name            varchar(60)                   ,
          cc_street_type            char(15)                      ,
          cc_suite_number           char(10)                      ,
          cc_city                   varchar(60)                   ,
          cc_county                 varchar(30)                   ,
          cc_state                  char(50)                       ,
          cc_zip                    char(20)                      ,
          cc_country                varchar(20)                   ,
          cc_gmt_offset             decimal(5,2)                  ,
          cc_tax_percentage         decimal(5,2)                  
        )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name4}
        (
            web_site_sk               integer               not null,
            web_site_id               char(16)              not null,
            web_rec_start_date        date                          ,
            web_rec_end_date          date                          ,
            web_name                  varchar(50)                   ,
            web_open_date_sk          integer                       ,
            web_close_date_sk         integer                       ,
            web_class                 varchar(50)                   ,
            web_manager               varchar(40)                   ,
            web_mkt_id                integer                       ,
            web_mkt_class             varchar(50)                   ,
            web_mkt_desc              varchar(100)                  ,
            web_market_manager        varchar(100)                   ,
            web_company_id            integer                  ,
            web_company_name          char(50)                      ,
            web_street_number         char(10)                      ,
            web_street_name           varchar(60)                   ,
            web_street_type           char(15)                      ,
            web_suite_number          char(50)                      ,
            web_city                  varchar(60)                   ,
            web_county                varchar(30)                   ,
            web_state                 char(20)                       ,
            web_zip                   char(50)                      ,
            web_country               varchar(20)                   ,
            web_gmt_offset            decimal(5,2)                  ,
            web_tax_percentage        decimal(5,2)                  
         )
         WITH (STORAGE_TYPE=USTORE) partition by range(web_rec_start_date)
        (
         partition p1 values less than('1990-01-01'),
         partition p2 values less than('1995-01-01'),
         partition p3 values less than('2000-01-01'),
         partition p4 values less than('2005-01-01'),
         partition p5 values less than('2010-01-01'),
         partition p6 values less than('2015-01-01'),
         partition p7 values less than(maxvalue)
        );
        create table {self.table_name5}
        (
            i_item_sk                 integer               not null,
            i_item_id                 char(16)              not null,
            i_rec_start_date          date                          ,
            i_rec_end_date            date                          ,
            i_item_desc               varchar(200)                  ,
            i_current_price           decimal(7,2)                  ,
            i_wholesale_cost          decimal(7,2)                  ,
            i_brand_id                integer                       ,
            i_brand                   char(50)                      ,
            i_class_id                integer                       ,
            i_class                   char(50)                      ,
            i_category_id             integer                       ,
            i_category                char(50)                      ,
            i_manufact_id             integer                       ,
            i_manufact                char(50)                      ,
            i_size                    char(20)                      ,
            i_formulation             char(20)                      ,
            i_color                   char(20)                      ,
            i_units                   char(10)                      ,
            i_container               char(10)                      ,
            i_manager_id              integer                       ,
            i_product_name            char(50)                      
        )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name6}
        (
            c_customer_sk             integer               not null,
            c_customer_id             char(16)              not null,
            c_current_cdemo_sk        integer                       ,
            c_current_hdemo_sk        integer                       ,
            c_current_addr_sk         integer                       ,
            c_first_shipto_date_sk    integer                       ,
            c_first_sales_date_sk     integer                       ,
            c_salutation              char(10)                      ,
            c_first_name              char(20)                      ,
            c_last_name               char(30)                      ,
            c_preferred_cust_flag     char(1)                       ,
            c_birth_day               integer                       ,
            c_birth_month             integer                       ,
            c_birth_year              integer                       ,
            c_birth_country           varchar(20)                   ,
            c_login                   char(13)                      ,
            c_email_address           char(50)                      ,
            c_last_review_date        char(10)
        )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name7}
        (
            ca_address_sk             integer               not null,
            ca_address_id             char(16)              not null,
            ca_street_number          char(10)                      ,
            ca_street_name            varchar(60)                   ,
            ca_street_type            char(15)                      ,
            ca_suite_number           char(10)                      ,
            ca_city                   varchar(60)                   ,
            ca_county                 varchar(30)                   ,
            ca_state                  char(2)                       ,
            ca_zip                    char(10)                      ,
            ca_country                varchar(20)                   ,
            ca_gmt_offset             decimal(5,2)                  ,
            ca_location_type          char(20)                     
        )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name8}
        (
            ss_sold_date_sk           integer                       ,
            ss_sold_time_sk           integer                       ,
            ss_item_sk                integer               not null,
            ss_customer_sk            integer                       ,
            ss_cdemo_sk               integer                       ,
            ss_hdemo_sk               integer                       ,
            ss_addr_sk                integer                       ,
            ss_store_sk               integer                       ,
            ss_promo_sk               integer                       ,
            ss_ticket_number          bigint               not null,
            ss_quantity               integer                       ,
            ss_wholesale_cost         decimal(7,2)                  ,
            ss_list_price             decimal(7,2)                  ,
            ss_sales_price            decimal(7,2)                  ,
            ss_ext_discount_amt       decimal(7,2)                  ,
            ss_ext_sales_price        decimal(7,2)                  ,
            ss_ext_wholesale_cost     decimal(7,2)                  ,
            ss_ext_list_price         decimal(7,2)                  ,
            ss_ext_tax                decimal(7,2)                  ,
            ss_coupon_amt             decimal(7,2)                  ,
            ss_net_paid               decimal(7,2)                  ,
            ss_net_paid_inc_tax       decimal(7,2)                  ,
            ss_net_profit             decimal(7,2)
         ) WITH (STORAGE_TYPE=USTORE);"""
        self.log.info(cmd_sql)
        create_result = self.pri_sh.execut_db_sql(cmd_sql)
        self.log.info(create_result)
        self.assertTrue(
            create_result.count(self.constant.TABLE_CREATE_SUCCESS) == 8,
            '执行失败:' + step_txt)

        step_txt = '-----step2:使用copy命令插入数据，expect: 数据插入成功-----'
        self.log.info(step_txt)
        text = '-----step2.1: 获取当前主机的IP地址;expect: 成功-----'
        self.log.info(text)
        cmd = "ifconfig -a|grep inet6 -a2|grep broadcast|awk '{print $2}'"
        self.log.info(cmd)
        self.cur_ip = subprocess.getoutput(cmd).strip()
        self.log.info(self.cur_ip)

        text = '-----step2.2: 复制文件到数据库主机对应目录 expect: 成功-----'
        self.log.info(text)
        if self.cur_ip == self.pri_sh.node.ssh_host:
            self.log.info('当前主机是数据库主机，无需远程复制')
            cmd = f"cp {self.cur_path_1} {self.parent_path};" \
                f"cp {self.cur_path_2} {self.parent_path};" \
                f"cp {self.cur_path_3} {self.parent_path};" \
                f"cp {self.cur_path_4} {self.parent_path};" \
                f"cp {self.cur_path_5} {self.parent_path};" \
                f"cp {self.cur_path_6} {self.parent_path};" \
                f"cp {self.cur_path_7} {self.parent_path};" \
                f"cp {self.cur_path_8} {self.parent_path};"
            self.log.info(cmd)
            res = self.pri_sh.node.sh(cmd).result()
            self.log.info(res)
        else:
            self.com.scp_file(self.pri_root, self.copy_file1,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file2,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file3,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file4,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file5,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file6,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file7,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file8,
                              self.parent_path)
            chmod_cmd = f"chmod 777 -R {self.parent_path};" \
                        f"ls -al {self.parent_path}"
            self.log.info(chmod_cmd)
            chmod_result = self.pri_root.sh(chmod_cmd).result()
            self.log.info(chmod_result)

        text = '-----step2.3: 查看数据库主机目标路径下文件 expect: 成功-----'
        self.log.info(text)
        cmd = f"ls -l {self.parent_path}| grep -E '{self.copy_file1}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file2}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file3}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file4}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file5}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file6}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file7}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file8}';"
        self.log.info(cmd)
        res = self.pri_root.sh(cmd).result()
        self.log.info(res)
        self.assertNotIn('No such file or directory', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file1}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file2}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file3}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file4}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file5}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file6}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file7}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file8}', res, '文件拷贝失败')

        text = '-----step2.4:使用copy命令插入数据，expect: 数据插入成功-----'
        self.log.info(text)
        self.log.info('---------------使用copy命令插入数据------------------')
        copy_sql = f"copy {self.table_name1} from '{self.target_path1}' " \
            f"delimiters ',';" \
            f"copy {self.table_name2} from '{self.target_path2}' " \
            f"delimiters ',';" \
            f"copy {self.table_name3} from '{self.target_path3}' " \
            f"delimiters ',';" \
            f"copy {self.table_name4} from '{self.target_path4}' " \
            f"delimiters ',';" \
            f"copy {self.table_name5} from '{self.target_path5}' " \
            f"delimiters ',';" \
            f"copy {self.table_name6} from '{self.target_path6}' " \
            f"delimiters ',';" \
            f"copy {self.table_name7} from '{self.target_path7}' " \
            f"delimiters ',';" \
            f"copy {self.table_name8} from '{self.target_path8}' " \
            f"delimiters ',';"
        self.log.info(copy_sql)
        copy_result = self.pri_sh.execut_db_sql(copy_sql)
        self.log.info(copy_result)
        self.assertIn('COPY 12', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 50000', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 18000', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 10000', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 6', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 30', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 2880404', copy_result, '执行失败:' + step_txt)

        step_txt = '-----step3: 子查询调优subquery—union/minus/union all混合使用' \
                   ' expect: 成功-----'
        self.log.info(step_txt)
        cmd_sql = f"""
select sum(i_current_price), avg(i_manager_id) 
  from {self.table_name5} 
 where (i_item_sk,1) not in 
       (select (select distinct (s_market_id)
                  from {self.table_name1} t1 
                 where s_store_sk = i_item_sk 
                   and t1.s_store_id = t2.s_store_id) ,1
          from {self.table_name1} t2 
         where s_market_id < 10 
           and s_store_sk = i_category_id 
        minus 
        select max(cc_mkt_id + 21) ,2
          from {self.table_name3} 
         where cc_class like '%a%' 
           and cc_call_center_sk > i_item_sk 
        union all 
        select s_gmt_offset ,''
          from {self.table_name1}
         where extract(day from  
                         case 
                           when s_rec_start_date = s_rec_end_date then 
                            s_rec_start_date 
                           else 
                            s_rec_end_date 
                         end) = 13 
           and s_rec_start_date < i_rec_end_date 
		   and exists(select s_gmt_offset from store where s_market_id = i_manager_id)
        union 
        select coalesce(c_birth_day, 1) - 10 ,''
          from {self.table_name6} 
         inner join {self.table_name2} 
            on c_birth_month = web_mkt_id 
           and c_birth_day = extract(day from web_rec_start_date) 
           and extract(year from web_rec_start_date) = 
               extract(year from i_rec_start_date) 
         where length(web_name) < 17 
           and c_birth_country like '%J%') 
   and i_item_sk < 1000 
 order by 1, 2;"""
        self.log.info(cmd_sql)
        query_result = self.pri_sh.execut_db_sql(cmd_sql)
        self.log.info(query_result)

    def tearDown(self):
        step_txt = '-----step4: 环境清理 expect:清理成功-----'
        self.log.info(step_txt)
        self.log.info('删除表')
        cmd_sql = self.pri_sh.execut_db_sql(f"""
                    drop table {self.table_name1} cascade;
                    drop table {self.table_name2} cascade;
                    drop table {self.table_name3} cascade;
                    drop table {self.table_name4} cascade;
                    drop table {self.table_name5} cascade;
                    drop table {self.table_name6} cascade;
                    drop table {self.table_name7} cascade;
                    drop table {self.table_name8} cascade;""")
        self.log.info(cmd_sql)
        text = '-----删除文件-----'
        self.log.info(text)
        rm_cmd = f"rm -rf {self.target_path1};" \
            f"rm -rf {self.target_path2};" \
            f"rm -rf {self.target_path3};" \
            f"rm -rf {self.target_path4};" \
            f"rm -rf {self.target_path5};" \
            f"rm -rf {self.target_path6};" \
            f"rm -rf {self.target_path7};" \
            f"rm -rf {self.target_path8};"
        self.log.info(rm_cmd)
        self.com.get_sh_result(self.pri_root, rm_cmd)
        for s in [self.target_path1, self.target_path2]:
            cmd = f'if [ -f {s} ]; then echo "does exists"; ' \
                f'else echo "not exists"; fi'
            file_judgments = self.com.get_sh_result(self.pri_root, cmd)
            self.assertIn('not exists', file_judgments, "清理失败" + text)
        self.assertTrue(
            cmd_sql.count(self.constant.DROP_TABLE_SUCCESS) == 8,
            '执行失败:' + step_txt)
        self.log.info(f'----- {os.path.basename(__file__)} end-----')
